{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: No level 1 heading here so that the numbering starting with ''Introduction'' below matches the manuscript."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%reset -f  # gives \"invalid alias\" errors\n",
    "%config InlineBackend.rc = {}\n",
    "%config InlineBackend.figure_format = \"retina\"\n",
    "%matplotlib inline\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import io\n",
    "import pickle\n",
    "from pprint import pprint\n",
    "\n",
    "import matplotlib.dates as mpl_d\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mpl_t\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import sklearn.linear_model\n",
    "\n",
    "import IPython.display\n",
    "\n",
    "import experiment as xp\n",
    "import google_health_trends as ght\n",
    "import ili\n",
    "import models\n",
    "import noise\n",
    "import wikipedia as wp\n",
    "import u\n",
    "\n",
    "np.random.seed(u.random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plt_timeseries_setup(ax=None):\n",
    "   if (ax is None):\n",
    "      ax = plt.gca()\n",
    "   ax.set_xlim(\"2011-Jun-5\",\"2016-Jul-24\")\n",
    "   ax.xaxis.set_major_locator(mpl_d.YearLocator())\n",
    "   #ax.tick_params(\"x\", direction=\"in\", length=10.0)\n",
    "   ax.set_xticklabels(ax.get_xticks(), ha=\"left\")\n",
    "   #ax.xaxis.set_major_formatter(mpl_d.DateFormatter(\"%Y-%b-%d\"))\n",
    "   ax.xaxis.set_major_formatter(mpl_d.DateFormatter(\"%Y\"))\n",
    "   ax.xaxis.set_minor_locator(mpl_d.MonthLocator(bymonth=(3,5,7,9,11)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.palplot(u.colors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "ili.load()\n",
    "wp.load()\n",
    "ght.load()\n",
    "with open(u.datapath_tl / \"out.pickle\", \"rb\") as fp:\n",
    "   out = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sanity check of output contents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top level dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pprint(sorted(out.keys()))\n",
    "assert (out.keys() == {\"decept\",\n",
    "                       \"features\",\n",
    "                       \"lambda_cv_means_bests\",\n",
    "                       \"lambda_cv_means_bias\",\n",
    "                       \"lambda_cv_selected\",\n",
    "                       \"models\",\n",
    "                       \"results\",\n",
    "                       \"synthetic_draws\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Input features themselves. Dictionary.\n",
    "\n",
    "* keys: name of feature class\n",
    "* values: DataFrame of all features of that type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pprint([(k, len(v.columns), len(v.index), type(v))\n",
    "        for k, v in out[\"features\"].items()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### decept"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deceptiveness of each feature. Nested dictionary.\n",
    "\n",
    "* keys:\n",
    "  * name of feature class\n",
    "  * deceptiveness noisiness\n",
    "* values: Series, deceptiveness of each feature in that class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (fn, fn_d) in out[\"decept\"].items():\n",
    "   for (dn, s) in fn_d.items():\n",
    "      print(\"(%s, %0.2f): %d, %s\" % (fn, dn, len(s.index), type(s)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### lambda cross-validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List of best lambdas:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (out[\"lambda_cv_means_bests\"] is not None):\n",
    "   print(type(out[\"lambda_cv_means_bests\"]), len(out[\"lambda_cv_means_bests\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bias function added to each model's lambda RMSE:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (out[\"lambda_cv_means_bias\"] is not None):\n",
    "   out[\"lambda_cv_means_bias\"].plot(lw=1, logx=True); None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Selected lambda (mean of the best for each model):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(xp.lambda_cv_min, xp.lambda_cv_max, xp.lambda_cv_step_ct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out[\"lambda_cv_selected\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All the models we fitted. Nested dictionary.\n",
    "\n",
    "* keys:\n",
    "  * name of input feature class\n",
    "  * training start\n",
    "  * deceptiveness noise\n",
    "  * model class name\n",
    "* values: model objects"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check consistency. Features and deceptiveness in models should be references, not copies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert set(xp.f_feature_name) == out[\"models\"].keys()\n",
    "for (fn, fn_d) in out[\"models\"].items():\n",
    "   assert set(xp.f_training_start) == fn_d.keys()\n",
    "   for (ts, ts_d) in fn_d.items():\n",
    "      assert set(xp.f_decept_noise) == ts_d.keys()\n",
    "      for (dn, dn_d) in ts_d.items():\n",
    "         assert {mc.name for mc in xp.f_model_class} == dn_d.keys()\n",
    "         for (mn, m) in dn_d.items():\n",
    "            assert fn == m.features_name\n",
    "            assert ts == m.training_start\n",
    "            assert dn == m.decept_noise\n",
    "            assert mn == m.name\n",
    "            assert id(m.features) == id(out[\"features\"][m.features_name])\n",
    "            assert id(m.decept) == id(out[\"decept\"][m.features_name][dn])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert {ef.__name__ for ef in xp.e_functions} == out[\"results\"].keys()\n",
    "pprint(sorted(out[\"results\"].keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check rows and columns of results dataframes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in out[\"results\"].values():\n",
    "   assert set(df.columns.get_level_values(0)) == set(xp.f_feature_name)\n",
    "   assert (   set(df.columns.get_level_values(1))\n",
    "           == set(mc.name for mc in xp.f_model_class))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### synthetic_draws"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the raw return values from `features_draw()` for the synthetic features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert type(out[\"synthetic_draws\"]) == list\n",
    "for i in out[\"synthetic_draws\"]:\n",
    "   assert len(i) == 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-distributable experiment dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I believe everything except the Google data can be redistributed. For now, just put the synthetic features in an Excel spreadsheet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out[\"features\"][\"synthetic\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out[\"features\"][\"synthetic\"].to_excel(\"synthetic.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**If we later want to scrub `out.pickle` instead**:\n",
    "2. change references to None\n",
    "3. grep around in the blob to make sure we didn't miss any references (e.g. with `pickletools` module)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfeat = ili.us_national.copy() \\\n",
    "        + 0.30 * noise.normal(ili.us_national.index) \\\n",
    "        + 8.00 * noise.linear(ili.us_national.index, \"2014-07-06\")\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(ili.us_national, color=\"gray\", lw=2.0,\n",
    "         label=\"ILI\")\n",
    "plt.plot(dfeat, color=u.colors[0], lw=1.0,\n",
    "         label=\"model\")\n",
    "\n",
    "plt.ylabel(\"ILI %\")\n",
    "plt.ylim(0,11.5)\n",
    "\n",
    "plt_timeseries_setup()\n",
    "plt.legend()\n",
    "plt.savefig(\"us_ili_synthetic_deceptive.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mq = pd.read_csv(u.datapath_tl / \"misc.ght.csv\",\n",
    "                 header=0, skiprows=[1,], index_col=0, parse_dates=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "\n",
    "#plt.plot(ili.us_national, color=\"black\", lw=1.5, label=\"ILI\")\n",
    "\n",
    "flu3 = ili.us_national[:\"2014-Jun-29\"]\n",
    "\n",
    "labeled=False\n",
    "for qs in (\"how long are you contagious\",\n",
    "           \"flu contagious\",\n",
    "           \"how long am i contagious\",\n",
    "           \"influenza contagious\",\n",
    "           \"when are you contagious\"):\n",
    "   query = mq[qs]\n",
    "   query3 = query[:\"2014-Jun-29\"]\n",
    "   reg = sklearn.linear_model.LinearRegression()\n",
    "   # .values.reshape() is to avoid two different deprecation warnings.\n",
    "   reg.fit(query3.values.reshape(-1,1), flu3)\n",
    "   query_normalized = reg.coef_[0] * query + reg.intercept_\n",
    "   if (labeled):\n",
    "      label=\"_nolabel_\"  # https://stackoverflow.com/a/35710894\n",
    "   else:\n",
    "      label=\"contagiousness searches\"\n",
    "      labeled=True\n",
    "   plt.plot(query_normalized, lw=0.5, color=u.colors[0], label=label)\n",
    "\n",
    "plt.plot(ili.us_national, color=\"gray\", lw=2.0, label=\"ILI\")\n",
    "plt.axvline(x=u.train_end, color=\"black\", lw=0.5)\n",
    "\n",
    "plt.ylabel(\"ILI %\")\n",
    "\n",
    "plt.legend()\n",
    "plt_timeseries_setup()\n",
    "plt.savefig(\"us_ili_and_deceptive.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data sources"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### U.S. influenza-like illness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "\n",
    "plt.plot(ili.us_national, color=\"gray\", lw=2.0)\n",
    "\n",
    "plt.ylabel(\"ILI %\")\n",
    "\n",
    "plt_timeseries_setup()\n",
    "plt.savefig(\"us_ili.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Flu-related concepts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(No figures in this section.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Web search volume"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Example features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "y1 = plt.axes()\n",
    "#y2 = plt.twinx()\n",
    "\n",
    "#y2.plot(ili.us_national, color=\"gray\", lw=1.5)\n",
    "y1.plot(ght.topics[\"Influenza A virus (Virus)\"], color=u.colors[2], lw=1.5,\n",
    "        label=\"topic: Influenza A virus (Virus)\")\n",
    "y1.plot(ght.raws[\"respiratory system\"], color=u.colors[0], lw=1.5,\n",
    "        label=\"raw query: respiratory system\")\n",
    "\n",
    "y1.set_ylabel(\"searches per million\")\n",
    "y1.set_ylim([0,38.5e-6])\n",
    "y1.yaxis.set_major_formatter(mpl_t.FuncFormatter(lambda y, pos: \"%g\" % (y*1e6)))\n",
    "\n",
    "plt_timeseries_setup()\n",
    "plt.legend(bbox_to_anchor=(0.48,0.68), labelspacing=0.3)\n",
    "plt.savefig(\"ght_examples.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Table of category distance and corresponding deceptiveness:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(i, ght.deceptiveness(i)) for i in range(1,8)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(ght.raw_decept.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(ght.topic_decept.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght.raw_decept.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght_raw_dist_hist = ght.raw_decept.groupby([\"distance\"])[\"distance\"].count()\n",
    "ght_raw_dist_hist.name = \"raw\"\n",
    "ght_raw_dist_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght_raw_decept_hist = ght.raw_decept.groupby([\"decept\"])[\"decept\"].count()\n",
    "ght_raw_decept_hist.name = \"raw\"\n",
    "ght_raw_decept_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght.topic_decept.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght_topic_dist_hist = ght.topic_decept.groupby([\"distance\"])[\"distance\"].count()\n",
    "ght_topic_dist_hist.name = \"topic\"\n",
    "ght_topic_dist_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght_topic_decept_hist = ght.topic_decept.groupby([\"decept\"])[\"decept\"].count()\n",
    "ght_topic_decept_hist.name = \"topic\"\n",
    "ght_topic_decept_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght_dist_hist = pd.concat([ght_raw_dist_hist, ght_topic_dist_hist],\n",
    "                          axis=1)\n",
    "ght_dist_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght_decept_hist = pd.concat([ght_raw_decept_hist, ght_topic_decept_hist],\n",
    "                            axis=1)\n",
    "ght_decept_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "buf = io.StringIO()\n",
    "buf.write(\"\"\"\\\n",
    "\\\\begin{tabular}{ccrr}\n",
    "\\\\toprule\n",
    "  \\\\textbf{Distance}\n",
    "& \\\\textbf{Deceptiveness}\n",
    "& \\\\textbf{Strings}\n",
    "& \\\\textbf{Topics} \\\\\\\\\n",
    "\\\\cmidrule(r){1-2}\n",
    "\\\\cmidrule(l){3-4}\n",
    "\"\"\")\n",
    "\n",
    "for r in ght_dist_hist.itertuples():\n",
    "   buf.write(\"%d & %0.02f & %d & %d \\\\\\\\ \\n\"\n",
    "             % (r.Index, ght.deceptiveness(r.Index), r.raw, r.topic))\n",
    "\n",
    "buf.write(\"\"\"\\\n",
    "\\\\bottomrule\n",
    "\\\\end{tabular}\n",
    "\"\"\")\n",
    "\n",
    "print(buf.getvalue())\n",
    "with open(\"ght_decept.tex\", \"w\") as fp:\n",
    "   fp.write(buf.getvalue())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Synthetic input features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basis functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(u.fig_width, 1.4*u.fig_width))\n",
    "\n",
    "def subplot_setup(ax, xlabels=False):\n",
    "   plt_timeseries_setup(ax)\n",
    "   ax.set_ylim(-0.15,1.15)\n",
    "   ax.yaxis.set_major_locator(mpl_t.FixedLocator([0,1]))\n",
    "   if (not xlabels):\n",
    "      ax.tick_params(\"x\", labelbottom=False)\n",
    "\n",
    "def subplot(loc, key, title=None, xlabels=False):\n",
    "   ax = plt.subplot(loc)\n",
    "   if (title is not None):\n",
    "      plt.title(title)\n",
    "   plt.plot(ili.us_national / ili.us_national.max(), color=\"gray\", lw=0.5)\n",
    "   plt.plot(xp.systematic_bases[key])\n",
    "   subplot_setup(ax, xlabels)\n",
    "      \n",
    "subplot(711, \"oprah annual\", \"Oprah effects: annual, fore, late\")\n",
    "subplot(712, \"oprah fore\")\n",
    "subplot(713, \"oprah late\", xlabels=True)\n",
    "\n",
    "subplot(714, \"drift steady\", \"Drift: steady, late\")\n",
    "subplot(715, \"drift late\", xlabels=True)\n",
    "\n",
    "subplot(716, \"cycle annual\", \"Cycle: annual, ending\")\n",
    "subplot(717, \"cycle ending\", xlabels=True)\n",
    "\n",
    "# FIXME: This doesn't actually yield variable spacing.\n",
    "# Maybe GridSpec or mpl_toolkits.axes_grid1?\n",
    "plt.tight_layout(pad=0, h_pad=0, w_pad=0)\n",
    "plt.savefig(\"bases.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fmt_weights(w):\n",
    "   s = (  \"%.2f / %.2f / %.2f (%.2f %.2f %.2f; %.2f %.2f; %.2f %.2f)\"\n",
    "        % ((w[0], w[1], w[2:].sum()) + tuple(w[2:])))\n",
    "   return s.replace(\"0.00\", \"0\")\n",
    "\n",
    "def plot_feature(f):\n",
    "   time_series = f[0]\n",
    "   weights = f[7]\n",
    "   plt.figure(figsize=(u.fig_width, 0.2*u.fig_width))\n",
    "   ax = plt.axes()\n",
    "   plt.title(\"%s: %s\" % (time_series.name, fmt_weights(weights)))\n",
    "   plt.plot(models.normalize(ili.us_national), color=\"gray\", lw=0.5)\n",
    "   plt.plot(models.normalize(time_series), lw=1)\n",
    "   #ax.set_ylim(-0.05,1.05)\n",
    "   plt.tick_params(\"x\", bottom=\"off\", labelbottom=False)\n",
    "   #plt.tick_params(\"y\", left=\"off\", labelleft=False)\n",
    "   plt.show()\n",
    "\n",
    "for f in sorted(out[\"synthetic_draws\"], key=lambda f: f[7][1:].sum())[:1000]:\n",
    "   plot_feature(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "\n",
    "plt.plot(out[\"features\"][\"synthetic\"][314], lw=1.5, label=\"least deceptive\")\n",
    "plt.plot(out[\"features\"][\"synthetic\"][386], lw=1.5, label=\"middling\")\n",
    "plt.plot(out[\"features\"][\"synthetic\"][412], lw=1.5, label=\"most deceptive\")\n",
    "\n",
    "plt.yticks([])\n",
    "\n",
    "plt_timeseries_setup()\n",
    "plt.legend() #bbox_to_anchor=(0.48,0.68), labelspacing=0.3)\n",
    "plt.savefig(\"synthetic_examples.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Histogram of deceptiveness"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We compute the histogram rather than plotting it directly so that we can put both types of features on the same plot."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = np.linspace(0, 1, 11)\n",
    "rounded = pd.cut(out[\"decept\"][\"synthetic\"][0.0], bins=bins, right=False,\n",
    "                 labels=(bins[:-1]))\n",
    "syn_hist = rounded.value_counts()\n",
    "syn_hist.sort_index(inplace=True)\n",
    "syn_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ght_decept_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "\n",
    "plt.plot(ght_decept_hist[\"raw\"], \"o-\", label=\"query strings\",\n",
    "         lw=1.0, ms=6, color=u.colors[0])\n",
    "plt.plot(ght_decept_hist[\"topic\"], \"o-\", label=\"topics\",\n",
    "         lw=1.0, ms=6, color=u.colors[1])\n",
    "plt.bar(list(syn_hist.index), list(syn_hist), label=\"synthetic\",\n",
    "        width=0.1, align=\"edge\",\n",
    "        color=u.colors[4], linewidth=0.0, edgecolor=\"black\")\n",
    "\n",
    "plt.xlabel(\"deceptiveness\")\n",
    "plt.xlim([-0.02,1.02])\n",
    "plt.ylabel(\"number of features\")\n",
    "plt.ylim([0,200])\n",
    "\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.savefig(\"decept_hist.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tinkering for us"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot all the models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is just for our reference, not the paper. We'll do better below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (fn, fn_d) in out[\"models\"].items():\n",
    "   plt.figure(figsize=(7,30))\n",
    "   plt.suptitle(fn, size=16)\n",
    "   sp_i = 0\n",
    "   for (ts_i, ts) in enumerate(fn_d.keys()):\n",
    "      ts_d = fn_d[ts]\n",
    "      for (dn_i, dn) in enumerate(ts_d.keys()):\n",
    "         dn_d = ts_d[dn]\n",
    "         sp_i += 1\n",
    "         plt.subplot(len(xp.f_decept_noise) * len(xp.f_training_start), 1, sp_i)\n",
    "         plt.title(\"ts %d / dn %g\" % (ts, dn))\n",
    "         plt.xticks([])\n",
    "         plt.yticks([])\n",
    "         plt.plot(ili.us_national, label=\"__nolabel__\",\n",
    "                  color=\"lightgray\", lw=3.0)\n",
    "         for m in dn_d.values():\n",
    "            if (m.name in (\"SMC\", \"SMM\", \"SMR\")): continue\n",
    "            plt.plot(m.prediction, label=m.name, lw=1.0)\n",
    "         plt.axvline(x=u.train_end, color=\"black\", lw=0.5)\n",
    "         plt.legend()\n",
    "   \n",
    "   plt.tight_layout(pad=0, h_pad=1, w_pad=1)\n",
    "   plt.subplots_adjust(top=0.963)\n",
    "   plt.savefig(\"all.%s.pdf\" % fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scatter plot of deceptivness vs. correlation during training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scatter plot of synthetic feature correlation during training period with ILI vs. deceptiveness. If deceptiveness is predicted well by correlation, then we don't need it, i.e., we probably sampled the synthetic features wrong."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(u.fig_width, u.fig_width))\n",
    "\n",
    "x = [ili.us_national_train.corr(f)\n",
    "     for (_, f) in out[\"features\"][\"synthetic\"].iteritems()]\n",
    "y = out[\"decept\"][\"synthetic\"][0]\n",
    "\n",
    "plt.scatter(x, y)\n",
    "\n",
    "plt.xlabel(\"correlation during training\")\n",
    "#plt.xlim(0,1)\n",
    "plt.ylabel(\"feature deceptiveness\")\n",
    "#plt.ylim(0,1)\n",
    "None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All results in big tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (m, df) in out[\"results\"].items():\n",
    "   print(m)\n",
    "   IPython.display.display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 𝝀"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out[\"lambda_cv_selected\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Histogram of best 𝝀 selections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(out[\"lambda_cv_means_bests\"], bins=100, log=True)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = pd.Series(out[\"lambda_cv_means_bests\"]).value_counts()\n",
    "h.sort_index(inplace=True)\n",
    "h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Each model’s 𝝀 plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (fn, fn_d) in out[\"models\"].items():\n",
    "   plt.figure(figsize=(7,30))\n",
    "   plt.suptitle(fn, size=16)\n",
    "   sp_i = 0\n",
    "   for (ts_i, ts) in enumerate(fn_d.keys()):\n",
    "      ts_d = fn_d[ts]\n",
    "      for (dn_i, dn) in enumerate(ts_d.keys()):\n",
    "         dn_d = ts_d[dn]\n",
    "         sp_i += 1\n",
    "         plt.subplot(len(xp.f_decept_noise) * len(xp.f_training_start), 1, sp_i)\n",
    "         plt.title(\"ts %d / dn %g\" % (ts, dn))\n",
    "         for m in dn_d.values():\n",
    "            if (m.name in (\"SMC\", \"SMM\", \"SMR\")): continue\n",
    "            plt.semilogx(m.lambda_cv_means, label=m.name, lw=1.0)\n",
    "         plt.axvline(x=out[\"lambda_cv_selected\"], color=\"black\", lw=0.5)\n",
    "         plt.ylim(0,1)\n",
    "         plt.legend()\n",
    "   \n",
    "   plt.tight_layout(pad=0, h_pad=1, w_pad=1)\n",
    "   plt.subplots_adjust(top=0.963)\n",
    "   plt.savefig(\"lambda.%s.pdf\" % fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selected models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(u.fig_width, 1.1*u.fig_width))\n",
    "\n",
    "for (i, fn) in enumerate(xp.f_feature_name):\n",
    "   plt.subplot(3, 1, i+1)\n",
    "   plt.title({'synthetic': 'synthetic',\n",
    "              'raw':       'query strings',\n",
    "              'topic':     'topics'}[fn])\n",
    "   \n",
    "   plt.plot(ili.us_national, label=\"ILI\", color=\"lightgray\", lw=4.0)\n",
    "\n",
    "   models_ = out[\"models\"][fn][0][0]\n",
    "   for m in models_.values():\n",
    "      plt.plot(m.prediction, label=m.name, lw=1.0)\n",
    "   \n",
    "   plt_timeseries_setup()\n",
    "   plt.ylim(0, 8.0)\n",
    "   if (i == 1):\n",
    "      plt.ylabel(\"ILI %\")\n",
    "   if (i != 2):\n",
    "      plt.xticks([])\n",
    "   plt.axvline(x=u.train_end, color=\"black\", lw=0.5)\n",
    "   if (i == 0):\n",
    "      plt.legend()\n",
    "\n",
    "plt.tight_layout(h_pad=1)\n",
    "plt.savefig(\"selected_models.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main results figure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This figure has nine subfigures. The various factors and metrics are reported as follows:\n",
    "\n",
    "* metrics: one figure per (metric, season pair)\n",
    "* season: see above\n",
    "* feature class: subplot rows\n",
    "* training start: subplot columns\n",
    "* decept noise: X axis\n",
    "* metric value: Y axis\n",
    "* model class: line style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These limits are all set so lower is better.\n",
    "limits = { (\"rmse\",               \"synthetic\"): ( 0.00,  3.00),\n",
    "           (\"rmse\",               \"raw\"):       ( 0.00,  0.90),\n",
    "           (\"rmse\",               \"topic\"):     ( 0.00,  0.90),\n",
    "           (\"r2\",                 \"synthetic\"): ( 1.00,  0.85),\n",
    "           (\"r2\",                 \"raw\"):       ( 1.00,  0.75),\n",
    "           (\"r2\",                 \"topic\"):     ( 1.00,  0.75),\n",
    "           (\"peak_timing_abs\",    \"synthetic\"): (-0.50, 12.50),\n",
    "           (\"peak_timing_abs\",    \"raw\"):       (-0.50, 12.50),\n",
    "           (\"peak_timing_abs\",    \"topic\"):     (-0.50, 12.50),\n",
    "           (\"peak_intensity_abs\", \"synthetic\"): ( 0.00,  2.10),\n",
    "           (\"peak_intensity_abs\", \"raw\"):       ( 0.00,  2.10),\n",
    "           (\"peak_intensity_abs\", \"topic\"):     ( 0.00,  2.10),\n",
    "           (\"hit_rate\",           \"synthetic\"): ( 1.00,  0.00),\n",
    "           (\"hit_rate\",           \"raw\"):       ( 1.00,  0.00),\n",
    "           (\"hit_rate\",           \"topic\"):     ( 1.00,  0.00) }\n",
    "\n",
    "rowheads = { \"synthetic\": \"synthetic\",\n",
    "             \"raw\": \"query strings\",\n",
    "             \"topic\": \"topics\" }\n",
    "\n",
    "def results_plot(metric, season):\n",
    "   \n",
    "   plt.figure(figsize=(u.fig_width, 0.9*u.fig_width))\n",
    "\n",
    "   sp_row_ct = len(xp.f_feature_name)\n",
    "   sp_col_ct = len(xp.f_training_start)\n",
    "   for (sp_row, fc) in enumerate(xp.f_feature_name):\n",
    "      for (sp_col, ts) in enumerate(xp.f_training_start):\n",
    "         plt.subplot(sp_row_ct, sp_col_ct, sp_col_ct * sp_row + sp_col + 1)\n",
    "         for (j, mc) in enumerate(i.name for i in xp.f_model_class):\n",
    "            # This garbage is the best indexing syntax I could come up with.\n",
    "            s = out[\"results\"][metric][fc,mc] \\\n",
    "                   .xs(season, level=\"test season\", axis=0) \\\n",
    "                   .xs(ts, level=\"training start\", axis=0)\n",
    "            if (mc == \"ridge\"):\n",
    "               plt.plot(s, \"-\",  lw=3, color=\"lightgray\", label=mc)\n",
    "            else:\n",
    "               plt.plot(s, \"o-\", lw=1, ms=3, color=u.colors[j], label=mc)\n",
    "         plt.tick_params(length=3, direction=\"out\")\n",
    "         plt.xlim(-0.03, 1.03)\n",
    "         if ((metric,fc) in limits):\n",
    "            plt.ylim(limits[(metric,fc)])\n",
    "         if (sp_row == 0):\n",
    "            titles = [\"training: 3 seasons\\n\",\n",
    "                      \"2 seasons\\n%s\" % rowheads[fc],\n",
    "                      \"1 season\\n\"]\n",
    "            if (sp_col == 0):\n",
    "               plt.legend(loc=\"upper left\", fontsize=\"small\",\n",
    "                          borderpad=0.2, labelspacing=0.3)\n",
    "         else:\n",
    "            titles = [\"\", rowheads[fc], \"\"]\n",
    "         plt.title(titles[ts], verticalalignment=\"bottom\")\n",
    "         if (sp_row == 2):\n",
    "            plt.xticks(xp.f_decept_noise, [\"0\", \"\", \"0.15\", \"0.4\", \"1\"])\n",
    "            if (sp_col == 1):\n",
    "               plt.xlabel(\"deceptiveness noise added\")\n",
    "         else:\n",
    "            plt.xticks(xp.f_decept_noise, (\"\" for i in xp.f_decept_noise))\n",
    "         if (sp_col == 0):\n",
    "            if (sp_row == 1):\n",
    "               plt.ylabel(\"%s : %s season\"\n",
    "                          % (metric.upper(), [\"4th\", \"5th\"][season-3]))\n",
    "         else:\n",
    "            plt.tick_params(axis=\"y\", labelleft=\"off\")\n",
    "\n",
    "   #plt.tight_layout(pad=0, h_pad=0, w_pad=0)\n",
    "   plt.tight_layout(w_pad=1)\n",
    "   plt.savefig(\"results_%s_%d.pdf\" % (metric, season))\n",
    "\n",
    "for metric in reversed(list(out[\"results\"].keys())):\n",
    "   for season in (4, 3):\n",
    "      results_plot(metric, season)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All data points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each column corresponds to one box in the box plot below. So, we should have 12 columns.\n",
    "\n",
    "rows:\n",
    "  - metric (RMSE, $r^2$, hit rate)\n",
    "  - deceptiveness noise (just 0, 0.05)\n",
    "  - training start\n",
    "  - test season\n",
    "\n",
    "columns:\n",
    "  - feature class\n",
    "  - algorithm, excluding ridge\n",
    "  \n",
    "elements:\n",
    "  - improvement over ridge (e.g. if it's twice as good, 2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out[\"results\"][\"hit_rate\"][\"synthetic\",\"linear fridge\"][0,3,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = [\"rmse\", \"r2\", \"hit_rate\"]\n",
    "dns = [0, 0.05]\n",
    "model_names = [i.name for i in xp.f_model_class if i != models.Ridge]\n",
    "# \"raw/topic\" is the raw results normalized against topic's ridge\n",
    "feature_names = [\"synthetic\", \"raw\", \"raw/topic\", \"topic\"]\n",
    "\n",
    "index = pd.MultiIndex.from_product([metrics, dns,\n",
    "                                    xp.f_training_start, xp.e_seasons],\n",
    "                                   names=[\"metric\", \"decept noise\",\n",
    "                                          \"training start\", \"season\"])\n",
    "columns = pd.MultiIndex.from_product([feature_names,model_names],\n",
    "                                     names=[\"features\", \"model\"])\n",
    "imp = pd.DataFrame(index=index, columns=columns, dtype=np.float64)\n",
    "\n",
    "# There is probably a fancy Pandas way to do this, but I'm tired and I\n",
    "# understand this loop.\n",
    "for metric in metrics:\n",
    "   for dn in dns:\n",
    "      for ts in xp.f_training_start:\n",
    "         for season in xp.e_seasons:\n",
    "            for fc in feature_names:\n",
    "               if (fc == \"raw/topic\"):\n",
    "                  n_fc = \"raw\"\n",
    "                  d_fc = \"topic\"\n",
    "               else:\n",
    "                  n_fc = fc\n",
    "                  d_fc = fc\n",
    "               ridge_perf = out[\"results\"][metric][d_fc,\"ridge\"][dn,season,ts]\n",
    "               for mn in model_names:\n",
    "                  perf = out[\"results\"][metric][n_fc,mn][dn,season,ts]\n",
    "                  if (metric in (\"r2\", \"hit_rate\")):\n",
    "                     # Measure of correlation: higher is better.\n",
    "                     # Compute the reduction in distance to 1 (perfect).\n",
    "                     ratio = (1-ridge_perf) / (1-perf)\n",
    "                  else:\n",
    "                     assert (metric in metrics)\n",
    "                     # Measure of error: lower is better.\n",
    "                     # Compute the reduction in error.\n",
    "                     ratio = ridge_perf / perf\n",
    "                  imp[fc,mn][metric,dn,ts,season] = ratio\n",
    "\n",
    "imp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RMSE summaries for body text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse = out[\"results\"][\"rmse\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ridge is same for every noise level, so just use 0.\n",
    "ridge = rmse.loc[0.00, (slice(None), \"ridge\")]\n",
    "ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.median(ridge.values), np.min(ridge.values), np.max(ridge.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gridge_low = rmse.loc[0:0.05, (slice(None), (\"threshold fridge\",\n",
    "                                             \"linear fridge\",\n",
    "                                             \"quadratic fridge\",\n",
    "                                             \"quartic fridge\"))]\n",
    "gridge_low"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(np.median(gridge_low.values),\n",
    " np.min(gridge_low.values),\n",
    " np.max(gridge_low.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gridge_high = rmse.loc[0.4:1.0, (slice(None), (\"threshold fridge\",\n",
    "                                               \"linear fridge\",\n",
    "                                               \"quadratic fridge\",\n",
    "                                               \"quartic fridge\"))]\n",
    "gridge_high"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(np.median(gridge_high.values),\n",
    " np.min(gridge_high.values),\n",
    " np.max(gridge_high.values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "rows:\n",
    "  - feature class\n",
    "  - algorithm\n",
    "  \n",
    "columns:\n",
    "  - max, median, min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_df = imp.min()\n",
    "median_df = imp.median()\n",
    "max_df = imp.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_names = [i.name for i in xp.f_model_class if i != models.Ridge]\n",
    "\n",
    "index = pd.MultiIndex.from_product([xp.f_feature_name, model_names],\n",
    "                                   names=[\"features\", \"model\"])\n",
    "columns = pd.MultiIndex.from_product([[\"min\", \"median\", \"max\"]],\n",
    "                                     names=[\"aggregate\"])\n",
    "summary = pd.DataFrame(index=index, columns=columns, dtype=np.float64)\n",
    "\n",
    "summary[\"min\"] = min_df\n",
    "summary[\"median\"] = median_df\n",
    "summary[\"max\"] = max_df\n",
    "\n",
    "summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Box plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "positions = { \"threshold fridge\": [ 1, 6,11,16],\n",
    "              \"linear fridge\":    [ 2, 7,12,17],\n",
    "              \"quadratic fridge\": [ 3, 8,13,18],\n",
    "              \"quartic fridge\":   [ 4, 9,14,19] }\n",
    "\n",
    "plt.figure(figsize=(u.fig_width, 1.3*u.fig_width))\n",
    "\n",
    "for (metric_i, metric) in enumerate([\"rmse\", \"r2\", \"hit_rate\"]): \n",
    "\n",
    "   plt.subplot(3, 1, metric_i + 1)\n",
    "   plt.title(metric)\n",
    "   \n",
    "   for (mc_i, mc) in enumerate(i.name for i in xp.f_model_class\n",
    "                               if i != models.Ridge):\n",
    "      df = imp.xs(metric, level=\"metric\", axis=0) \\\n",
    "              .xs(mc, level=\"model\", axis=1)\n",
    "      bp = plt.boxplot(df.T, positions=positions[mc], widths=0.7)\n",
    "      plt.setp(bp[\"boxes\"], color=u.colors[mc_i+1])\n",
    "      plt.setp(bp[\"whiskers\"], color=u.colors[mc_i+1], lw=0.7)\n",
    "      plt.setp(bp[\"caps\"], color=u.colors[mc_i+1], lw=0.7)\n",
    "      plt.setp(bp[\"medians\"], color=u.colors[mc_i+1], lw=0.7)\n",
    "\n",
    "   plt.axhline(y=1, color=\"black\", lw=0.5)\n",
    "   \n",
    "   plt.xlim(0, 20)\n",
    "   plt.xticks([2.5, 7.5, 12.5, 17.5],\n",
    "              [\"synthetic\", \"query string\",\n",
    "               \"string vs. topic ridge\", \"query topic\"])\n",
    "   plt.tick_params(length=0, axis=\"x\")\n",
    "\n",
    "   if (metric_i == 1):\n",
    "      plt.ylabel(\"times better than ridge\")\n",
    "   plt.ylim({\"rmse\":     (0,  5.0),\n",
    "             \"r2\":       (0, 10.0),\n",
    "             \"hit_rate\": (0,  2.5) }[metric])\n",
    "\n",
    "   if (metric_i == 0):\n",
    "      plt.plot([], color=u.colors[1], lw=1, label=\"threshold\")\n",
    "      plt.plot([], color=u.colors[2], lw=1, label=\"linear\")\n",
    "      plt.plot([], color=u.colors[3], lw=1, label=\"quadratic\")\n",
    "      plt.plot([], color=u.colors[4], lw=1, label=\"quartic\")\n",
    "      plt.legend(loc=\"upper right\")\n",
    "\n",
    "plt.tight_layout(h_pad=1)\n",
    "plt.savefig(\"summary_boxplot.pdf\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "12px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": "3",
   "toc_cell": false,
   "toc_position": {
    "height": "1142px",
    "left": "0px",
    "right": "845px",
    "top": "111px",
    "width": "170px"
   },
   "toc_section_display": "block",
   "toc_window_display": true,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
